package cn.imuduan;

import cn.imuduan.dao.MyBeanDao;
import cn.imuduan.pojo.MyBean;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.RedisScheduler;
import us.codecraft.webmagic.selector.Selectable;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * @author MuDuan
 * @date 2020/1/24 19:19
 */
public class myTest implements PageProcessor {

    private static Logger log = LoggerFactory.getLogger("WebMagicTest");
    private Site site = Site.me().setDomain("xiaohua.com").setSleepTime(1000).setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36");
    private static MyBeanDao myBeanDao;
    private boolean flag = false;

    static {
        log.info("Mybatis环境搭建中...");
        String resource = "mybatis-config.xml";
        try {
            InputStream inputStream = Resources.getResourceAsStream(resource);
            SqlSessionFactory sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);
            SqlSession sqlSession = sqlSessionFactory.openSession(true);
            myBeanDao = sqlSession.getMapper(MyBeanDao.class);
        } catch (IOException e) {
            e.printStackTrace();
            System.out.println("Mybatis环境搭建失败...");
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void process(Page page) {
        log.info("爬虫开始...");
        //这个地方可以实现先爬网站数据总量，再给爬虫加个定期执行任务实现自动更新数据效果
        if (!flag) {
            for (int i = 1; i < 1072; i++) {
                String nextUrl = "https://www.xiaohua.com/duanzi?page=" + (i + 1);
                page.addTargetRequest(nextUrl);
            }
            flag = true;
        }
        List<Selectable> nodes = page.getHtml().xpath("//div[@class='content-left']/div[@class='one-cont']").nodes(); //获取列表的条数
        Set<MyBean> set = new HashSet<>(); //使用set集合去重
        MyBean myBean = null;
        for (int i = 0; i < nodes.size(); i++) {
            myBean = new MyBean();
            myBean.setContent(page.getHtml().xpath("/html/body/div/div[8]/div[2]/div[2]/div[" + (i + 1) + "]/p/a/text()").get());   //内容
            myBean.setSupport(Integer.parseInt(page.getHtml().xpath("/html/body/div/div[8]/div[2]/div[2]/div[" + (i + 1) + "]/ul/li[1]/span/text()").get()));   //点赞数
            myBean.setNosupport(Integer.parseInt(page.getHtml().xpath("/html/body/div/div[8]/div[2]/div[2]/div[" + (i + 1) + "]/ul/li[2]/span/text()").get())); //差评数
            myBean.setCollect(Integer.parseInt(page.getHtml().xpath("/html/body/div/div[8]/div[2]/div[2]/div[" + (i + 1) + "]/ul/li[3]/span/text()").get()));   //收藏数
            myBean.setComment(Integer.parseInt(page.getHtml().xpath("/html/body/div/div[8]/div[2]/div[2]/div[" + (i + 1) + "]/ul/li[4]/a/span/text()").get())); //评论数
            myBean.setShares(Integer.parseInt(page.getHtml().xpath("/html/body/div/div[8]/div[2]/div[2]/div[" + (i + 1) + "]/ul/li[5]/span/text()").get()));    //分享数
            set.add(myBean);
        }
        //set转换为list
        List<MyBean> list = new ArrayList(set);
        //数据持久化
        myBeanDao.insertForeach(list);
    }

    public static void main(String[] args) {
        long startTime = System.currentTimeMillis();    //获取开始时间
        //使用redis实现多线程不爬取同个url，(实现增量爬取)
        JedisPool jedisPool = new JedisPool(new GenericObjectPoolConfig(), "47.95.14.28", 6379, 2000, "1278");
        Spider.create(new myTest()).addUrl("https://www.xiaohua.com/duanzi?page=1")
                .setScheduler(new RedisScheduler(jedisPool))
                .thread(10)
                .run();
        long endTime = System.currentTimeMillis();    //获取结束时间
        System.out.println("程序运行时间：" + (endTime - startTime) / 1000 + "s");    //输出程序运行时间
    }
}
